{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "id": "2aRnHGyN6SjR",
    "colab_type": "code",
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35.0
    },
    "outputId": "e78d20a1-5360-4976-9f2d-555c0e280729"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Hello World\n"
     ]
    }
   ],
   "source": [
    "print(\"Hello World\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "id": "P9nXM7106-SS",
    "colab_type": "code",
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 127.0
    },
    "outputId": "781b0fc9-d029-4c08-e6be-8265e0ee1137"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3Aietf%3Awg%3Aoauth%3A2.0%3Aoob&scope=email%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdocs.test%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdrive%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdrive.photos.readonly%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fpeopleapi.readonly&response_type=code\n",
      "\n",
      "Enter your authorization code:\n",
      "··········\n",
      "Mounted at /content/drive\n"
     ]
    }
   ],
   "source": [
    "from google.colab import drive\n",
    "drive.mount('/content/drive')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "id": "_vGGil-f7YeK",
    "colab_type": "code",
    "colab": {}
   },
   "outputs": [],
   "source": [
    "!cd drive/My\\ Drive/app"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "id": "K2oB0av18KJ9",
    "colab_type": "code",
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 269.0
    },
    "outputId": "59fcab82-1e19-4345-f4c1-7548292c0563"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                                       user                song  play_count\n0  4e11f45d732f4861772b2906f81a7d384552ad12  SOCKSGZ12A58A7CA4B           1\n1  4e11f45d732f4861772b2906f81a7d384552ad12  SOCVTLJ12A6310F0FD           1\n2  4e11f45d732f4861772b2906f81a7d384552ad12  SODLLYS12A8C13A96B           3\ndata_train:\n                                           user                song  \\\n7971   7bdfc45af7e15511d150e2acb798cd5e4788abf5  SOXBCZH12A67ADAD77   \n31459  c405c586f6d7aadbbadfcba5393b543fd99372ff  SOXFYTY127E9433E7D   \n14683  625d0167edbc5df88e9fbebe3fcdd6b121a316bb  SONOYIB12A81C1F88C   \n\n       play_count  \n7971            8  \n31459           3  \n14683           1  \ndata_val:\n                                           user                song  \\\n26019  3325fe1d8da7b13dd42004ede8011ce3d7cd205d  SOURVJI12A58A7F353   \n33943  e82b3380f770c78f8f067f464941057c798eaca2  SOKNWRZ12A8C13BF62   \n15356  bdfca47d03157d26f1404075172128a6f8a3d39e  SOMNGMO12A6702187E   \n\n       play_count  \n26019          63  \n33943          19  \n15356           4  \n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "dir_path = os.path.join(os.getcwd(), \"drive\", \"My Drive\", \"app\", \"data\")\n",
    "dir_path = \"./data/\"\n",
    "file_path = os.path.join(dir_path, \"triplet_dataset_sub.csv\")\n",
    "data = pd.read_csv(file_path)\n",
    "print(data.head(3))\n",
    "\n",
    "data_train, data_val = train_test_split(data, train_size=0.8, random_state=0)\n",
    "print(\"data_train:\\n{}\".format(data_train.head(3)))\n",
    "print(\"data_val:\\n{}\".format(data_val.head(3)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "id": "0YuwRQe29xH_",
    "colab_type": "code",
    "colab": {}
   },
   "outputs": [],
   "source": [
    "def init_user_song_playcount(df):\n",
    "    \"\"\"\n",
    "    初始化user和song对应的playcount\n",
    "    :param df: 输入DataFrame对象\n",
    "    :return: user,song对应的playcount值\n",
    "    \"\"\"\n",
    "    result_dict = dict()\n",
    "    values = df.values\n",
    "\n",
    "    for line in values:\n",
    "      user = line[0]\n",
    "      song = line[1]\n",
    "      play_count = line[2]\n",
    "\n",
    "      if user not in result_dict:\n",
    "        result_dict[user] = dict()\n",
    "      result_dict[user][song] = play_count\n",
    "\n",
    "    return result_dict\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "id": "2ArlSA-z967F",
    "colab_type": "code",
    "colab": {}
   },
   "outputs": [],
   "source": [
    "def init_user_songs(data):\n",
    "    \"\"\"\n",
    "    初始化user对应的songs列表\n",
    "    :param data: 原始User播放song记录\n",
    "    :return: \n",
    "    \"\"\"\n",
    "    user_songs = dict()\n",
    "    for each_user, each_value in data.groupby(by=\"user\"):\n",
    "        user_songs[each_user] = each_value.song.values\n",
    "    return user_songs\n",
    "\n",
    "\n",
    "def init_song_users(data):\n",
    "    \"\"\"\n",
    "    初始化song对应的users列表\n",
    "    :param data: 原始user播放song记录\n",
    "    :return: \n",
    "    \"\"\"\n",
    "    song_users = dict()\n",
    "    for each_song, each_value in data.groupby(by=\"song\"):\n",
    "        song_users[each_song] = each_value.user.values\n",
    "    \n",
    "    return song_users\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "id": "GZnDyhjb9-0h",
    "colab_type": "code",
    "colab": {}
   },
   "outputs": [],
   "source": [
    "def user_sim(user_1, user_2, user_songs):\n",
    "    \"\"\"\n",
    "    计算两个User的相似度\n",
    "    :param user_1: 1号User\n",
    "    :param user_2: 2号User\n",
    "    :return: 两个User的相似度\n",
    "    \"\"\"\n",
    "    songs_1_set = set(user_songs.get(user_1, list()))\n",
    "    songs_2_set = set(user_songs.get(user_2, list()))\n",
    "    intersection_num = len(songs_1_set.intersection(songs_2_set))\n",
    "    union_num = len(songs_1_set.union(songs_2_set))\n",
    "    \n",
    "    if union_num == 0:\n",
    "        return 0.0\n",
    "    else:\n",
    "        return 1.0 * intersection_num / union_num\n",
    "\n",
    "\n",
    "def song_sim(song_1, song_2, song_users):\n",
    "    \"\"\"\n",
    "    计算两首song的相似度\n",
    "    :param song_1: 1号song\n",
    "    :param song_2: 2号song\n",
    "    :param song_users: 原始User播放song的记录\n",
    "    :return: 两首song的相似度\n",
    "    \"\"\"\n",
    "    user_1_set = set(song_users.get(song_1, list()))\n",
    "    user_2_set = set(song_users.get(song_2, list()))\n",
    "    intersection_num = len(user_1_set.intersection(user_2_set))\n",
    "    union_num = len(user_1_set.union(user_2_set))\n",
    "    \n",
    "    if union_num == 0:\n",
    "        return 0.0\n",
    "    else:\n",
    "        return 1.0 * intersection_num / union_num\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "id": "hJFfllbm-BcL",
    "colab_type": "code",
    "colab": {}
   },
   "outputs": [],
   "source": [
    "def init_user_sim_matrix(user_songs):\n",
    "    \"\"\"\n",
    "    计算User之间的相似度矩阵, 以及user_id和user索引的映射关系\n",
    "    :param user_songs: User-播放过的songs\n",
    "    :return: User相似度矩阵, user_id--user索引的映射关系\n",
    "    \"\"\"\n",
    "    user_size = len(user_songs.keys())\n",
    "    user_sim_matrix = np.zeros(dtype=np.float, shape=(user_size, user_size))\n",
    "    user_to_index = dict()\n",
    "    \n",
    "    user_ids = list(user_songs.keys())\n",
    "    \n",
    "    for i, user_id in enumerate(user_ids):\n",
    "        user_to_index[user_id] = i\n",
    "        \n",
    "        for j in range(i + 1, len(user_ids)):\n",
    "            each_sim = user_sim(user_id, user_ids[j], user_songs)\n",
    "            user_sim_matrix[i, j] = each_sim\n",
    "            user_sim_matrix[j, i] = each_sim\n",
    "            \n",
    "    return user_sim_matrix, user_to_index\n",
    "\n",
    "\n",
    "def init_song_sim_matrix(song_users):\n",
    "    \"\"\"\n",
    "    计算song之间的相似度矩阵，以及song和song索引的映射关系\n",
    "    :param song_users: song--播放过song的users\n",
    "    :return: song相似度矩阵, song和song索引的映射关系\n",
    "    \"\"\"\n",
    "    song_size = len(song_users.keys())\n",
    "    song_sim_matrix = np.zeros(dtype=np.float, shape=(song_size, song_size))\n",
    "    song_to_index = dict()\n",
    "    \n",
    "    songs = list(song_users.keys())\n",
    "    \n",
    "    for i, song in enumerate(songs):\n",
    "        song_to_index[song] = i\n",
    "        \n",
    "        for j in range(i + 1, len(songs)):\n",
    "            each_sim = song_sim(song, songs[j], song_users)\n",
    "            song_sim_matrix[i, j] = each_sim\n",
    "            song_sim_matrix[j, i] = each_sim\n",
    "            \n",
    "    return song_sim_matrix, song_to_index\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "id": "rK478z5r-I91",
    "colab_type": "code",
    "colab": {}
   },
   "outputs": [],
   "source": [
    "import time\n",
    "\n",
    "\n",
    "def pred_user_cf(user, song, user_song_playcount, played_users_list, user_mean_scores, \n",
    "                 user_sim_matrix, user_to_index, df_train):\n",
    "    \"\"\"\n",
    "    预测user对song的打分\n",
    "    :param user: 目标user\n",
    "    :param song: 目标song\n",
    "    :param played_users_list: 播放过目标song的user列表\n",
    "    :param user_mean_scores: 每个user的平均打分\n",
    "    :param user_sim_matrix: 所有user间的相似度矩阵\n",
    "    :param user_to_index: user编号-user索引映射\n",
    "    :param df_train: 训练集数据\n",
    "    :return: \n",
    "    \"\"\"\n",
    "    pred_play_count = 0.0\n",
    "    total_weighted_count = 0.0\n",
    "    total_sim = 0.0\n",
    "    for each_user in played_users_list:\n",
    "        each_score = user_song_playcount[each_user][song]\n",
    "        each_sim = user_sim_matrix[user_to_index[each_user],\n",
    "                      user_to_index[user]]\n",
    "        mean_score = user_mean_scores[each_user]\n",
    "        total_weighted_count += each_sim * (each_score - mean_score)\n",
    "        total_sim += each_sim\n",
    "    \n",
    "    if abs(total_sim) < 1e-6:\n",
    "        pred_play_count = 0.0\n",
    "    else:\n",
    "        pred_play_count = total_weighted_count / total_sim\n",
    "    \n",
    "    return pred_play_count\n",
    "\n",
    "\n",
    "def recommend_user_cf(user, user_song_playcount, song_users, user_mean_scores,\n",
    "                      user_sim_matrix, user_to_index, df_train):\n",
    "    \"\"\"\n",
    "    为指定user推荐songs\n",
    "    :param user: 目标user\n",
    "    :param song_users: 每个song-播放过该song的users映射\n",
    "    :param user_mean_scores: 每个user的平均打分\n",
    "    :param user_sim_matrix: 所有user之间的相似度矩阵\n",
    "    :param user_to_index: 每个user-对应的user索引映射\n",
    "    :param df_train: 训练集数据DataFrame\n",
    "    :return: 推荐结果\n",
    "    \"\"\"\n",
    "    rtn_songs = []\n",
    "    for song, played_users in song_users.items():\n",
    "        if user in played_users:\n",
    "            continue\n",
    "\n",
    "        each_time_1 = time.time()\n",
    "        played_users_list = sorted(list(played_users), \n",
    "                                   key=lambda user_x:\n",
    "                                   user_sim_matrix[user_to_index[user_x],\n",
    "                                                   user_to_index[user]],\n",
    "                                   reverse=True)\n",
    "        if len(played_users_list) > 10:\n",
    "            played_users_list = played_users_list[0:10]\n",
    "        \n",
    "        each_time_1 = time.time()\n",
    "        pred_count = pred_user_cf(user, song, user_song_playcount, played_users_list, \n",
    "                                  user_mean_scores, user_sim_matrix, \n",
    "                                  user_to_index, df_train)\n",
    "        rtn_songs.append((pred_count, song))\n",
    "\n",
    "    return sorted(rtn_songs, key=lambda x: x[0], reverse=True)\n",
    "\n",
    "\n",
    "def recommend_all_user_cf(df_train, df_val, rec_num):\n",
    "    \"\"\"\n",
    "    基于User的CF方法: 根据训练集的数据，对所有校验集中的User推荐Song\n",
    "    :param df_train: 训练集\n",
    "    :param df_val: 校验集\n",
    "    :param rec_num: 为每个User推荐的song数量\n",
    "    :return: \n",
    "    \"\"\"\n",
    "    result_dict = dict()\n",
    "    \n",
    "    print(\"\\n开始初始化user_songs映射\")\n",
    "    time_start = time.time()\n",
    "    user_songs = init_user_songs(df_train)\n",
    "    print(\"初始化user_songs映射完成，耗时{:.4f}秒\".format(time.time() - time_start))\n",
    "    \n",
    "    print(\"\\n开始初始化song_users映射\")\n",
    "    time_start = time.time()\n",
    "    song_users = init_song_users(df_train)\n",
    "    print(\"初始化song_users映射完成，耗时{:.4f}秒\".format(time.time() - time_start))\n",
    "    \n",
    "    print(\"\\n开始初始化user_song_playcount映射\")\n",
    "    time_start = time.time()\n",
    "    user_song_playcount = init_user_song_playcount(df_train)\n",
    "    print(\"初始化user_song_playcount映射完成，耗时{:.4f}秒\".format(time.time() - time_start))\n",
    "\n",
    "    print(\"\\n开始初始化user_mean_scores映射\")\n",
    "    time_start = time.time()\n",
    "    user_mean_scores = dict()\n",
    "    for user, songs in user_songs.items():\n",
    "        user_mean_scores[user] = np.mean([user_song_playcount[user][song] for song in songs])\n",
    "    print(\"初始化user_mean_scores映射完成，耗时{:.4f}秒\".format(time.time() - time_start))\n",
    "    \n",
    "    print(\"\\n开始初始化user_sim_matrix, user_to_index\")\n",
    "    time_start = time.time()\n",
    "    user_sim_matrix, user_to_index = init_user_sim_matrix(user_songs)\n",
    "    print(\"初始化user_sim_matrix, user_to_index完成，耗时{:.4f}秒\".format(\n",
    "        time.time() - time_start))\n",
    "    \n",
    "    val_users = df_val[\"user\"].unique()\n",
    "    print(\"\\n开始为{}个来自校验集的user推荐songs\".format(len(val_users)))\n",
    "    time_start = time.time()\n",
    "    for i, user in enumerate(val_users):\n",
    "        if user not in user_songs:\n",
    "            print(\"{} is a new user.\".format(user))\n",
    "            continue\n",
    "\n",
    "        rec_songs = recommend_user_cf(user, user_song_playcount, song_users, user_mean_scores,\n",
    "                        user_sim_matrix, user_to_index, \n",
    "                        df_train)\n",
    "        if len(rec_songs) > 0:\n",
    "            result_dict[user] = rec_songs[0: rec_num]\n",
    "\n",
    "        if (i + 1) % 100 == 0:\n",
    "            print(\"已经为{}个user计算好了推荐song的结果，耗时{:.4f}秒\".format(\n",
    "                i + 1, time.time() - time_start\n",
    "            ))\n",
    "\n",
    "    print(\"为{}个来自校验集的user推荐songs的工作已完成，耗时{:.4f}秒\".format(\n",
    "        len(val_users), time.time() - time_start\n",
    "    ))\n",
    "\n",
    "    return result_dict, len(song_users)\n",
    "\n",
    "\n",
    "def evaluate(user_rec_dict, total_song_num, df_val):\n",
    "    \"\"\"\n",
    "    评估在校验集上的预测结果\n",
    "    :param user_rec_dict: 预测结果<user-推荐的song和评分列表>\n",
    "    :param total_song_num: 训练集中的song总数\n",
    "    :param df_val: 校验集\n",
    "    :return: \n",
    "    \"\"\"\n",
    "    n_tp = 0         # 校验集和预测结果匹配的song数\n",
    "    n_total_rec = 0      # 预测结果里的song数\n",
    "    n_total_real = 0      # 校验集里的song数\n",
    "    total_rec_songs = set()   # 预测结果里所有的song\n",
    "    rss_val = 0.0         # 预测结果与校验集结果的平方误差(用播放次数差来计算)\n",
    "    \n",
    "    unique_users = df_val[\"user\"].unique()\n",
    "    for user in unique_users:\n",
    "        real_songs = df_val.loc[df_val.user == user].song.values\n",
    "        n_total_real = n_total_real + len(real_songs)\n",
    "        \n",
    "        if user in user_rec_dict:\n",
    "            rec_songs = [x[1] for x in user_rec_dict[user]]\n",
    "            hit_songs = set(real_songs).intersection(set(rec_songs))\n",
    "        \n",
    "            total_rec_songs = total_rec_songs.union(hit_songs)\n",
    "            n_tp = n_tp + len(hit_songs)\n",
    "        \n",
    "            n_total_rec = n_total_rec + len(rec_songs)\n",
    "            \n",
    "            for pred_score, song in user_rec_dict[user]:\n",
    "                val_score = df_val.loc[df_val.user == user][df_val.song == song]\\\n",
    "                    .play_count.values\n",
    "                if len(val_score) > 0:\n",
    "                    rss_val = rss_val + (pred_score - val_score[0]) ** 2\n",
    "    \n",
    "    precision = 1.0 * n_tp / n_total_rec\n",
    "    recall = 1.0 * n_tp / n_total_real\n",
    "    coverage = 1.0 * len(total_rec_songs) / total_song_num\n",
    "    rmse = np.sqrt(1.0 * rss_val / df_val.shape[0])\n",
    "    \n",
    "    print(\"评价结束: precision={:.4f}, recall={:.4f}, coverage={:.4f}, rmse={:.4f}\".format(\n",
    "        precision, recall, coverage, rmse\n",
    "    ))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "id": "-1rcnLaY-SCo",
    "colab_type": "code",
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 577.0
    },
    "outputId": "eb4babb5-dc00-4624-b990-5b832dfbae6b"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n开始初始化user_songs映射\n初始化user_songs映射完成，耗时0.1950秒"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n\n开始初始化song_users映射\n初始化song_users映射完成，耗时0.2000秒"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n\n开始初始化user_song_playcount映射\n初始化user_song_playcount映射完成，耗时0.0250秒\n\n开始初始化user_mean_scores映射\n初始化user_mean_scores映射完成，耗时0.0280秒\n\n开始初始化user_sim_matrix, user_to_index\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "初始化user_sim_matrix, user_to_index完成，耗时3.4462秒\n\n开始为726个来自校验集的user推荐songs\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "已经为100个user计算好了推荐song的结果，耗时4.3572秒\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "已经为200个user计算好了推荐song的结果，耗时8.6565秒\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "已经为300个user计算好了推荐song的结果，耗时13.1628秒\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "已经为400个user计算好了推荐song的结果，耗时17.8690秒\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "已经为500个user计算好了推荐song的结果，耗时24.2054秒\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "已经为600个user计算好了推荐song的结果，耗时29.3507秒\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "67c5b5b1982902d15badd8ce0c18b3278ec4bfc0 is a new user.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "62420be0fd0df5ab0eb4cba35a4bc7cb3e3b506a is a new user.\n3ab78e39bddeaeb789edad041fff03050077417c is a new user.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "已经为700个user计算好了推荐song的结果，耗时34.8180秒\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "为726个来自校验集的user推荐songs的工作已完成，耗时36.3211秒\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Python37\\lib\\site-packages\\ipykernel_launcher.py:164: UserWarning: Boolean Series key will be reindexed to match DataFrame index.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "评价结束: precision=0.0296, recall=0.0285, coverage=0.0650, rmse=11.5904\n"
     ]
    }
   ],
   "source": [
    "# 为每个校验集中的User推荐10首Song\n",
    "result_dict, total_song_num = recommend_all_user_cf(data_train, data_val, 10)\n",
    "# 评价推荐结果\n",
    "evaluate(result_dict, total_song_num, data_val)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "id": "huLQXZjNItYk",
    "colab_type": "code",
    "colab": {}
   },
   "outputs": [],
   "source": [
    "import time\n",
    "\n",
    "\n",
    "def pred_item_cf(user, song, user_song_playcount, user_played_songs, \n",
    "                 song_sim_matrix, song_to_index, df_train):\n",
    "    \"\"\"\n",
    "    预测user对song的打分(CF方法的核心函数, 这里要基于song的相似性)\n",
    "    :param user: 目标user\n",
    "    :param song: 目标song\n",
    "    :param user_song_playcount: user-song-playcount映射\n",
    "    :param user_played_songs: 目标user播放过的song列表\n",
    "    :param song_sim_matrix: 所有song间的相似度矩阵\n",
    "    :param song_to_index: song编号-song索引映射\n",
    "    :param df_train: 训练集数据\n",
    "    :return: \n",
    "    \"\"\"\n",
    "    pred_play_count = 0.0\n",
    "    total_weighted_count = 0.0\n",
    "    total_sim = 0.0\n",
    "    for each_song in user_played_songs:\n",
    "        each_score = user_song_playcount[user][each_song]\n",
    "        each_sim = song_sim_matrix[song_to_index[each_song],\n",
    "                      song_to_index[song]]\n",
    "        total_weighted_count += each_sim * each_score\n",
    "        total_sim += each_sim\n",
    "    \n",
    "    if abs(total_sim) < 1e-6:\n",
    "        pred_play_count = 0.0\n",
    "    else:\n",
    "        pred_play_count = total_weighted_count / total_sim\n",
    "    \n",
    "    return pred_play_count\n",
    "\n",
    "\n",
    "def recommend_item_cf(user, song_users, user_song_playcount, user_played_songs,\n",
    "                      song_sim_matrix, song_to_index, df_train):\n",
    "    \"\"\"\n",
    "    为目标user推荐songs\n",
    "    :param user: 目标User\n",
    "    :param song_users: 每首song-播放过它的user列表的映射\n",
    "    :param user_song_playcount: user-song-playcount映射\n",
    "    :param user_played_songs: 目标User播放过的songs\n",
    "    :param song_sim_matrix: 所有song之间的相似度矩阵\n",
    "    :param song_to_index: song编号-song索引的映射\n",
    "    :param df_train: 训练数据集\n",
    "    \"\"\"\n",
    "    rtn_songs = []\n",
    "    for song in song_users.keys():\n",
    "      if song in user_played_songs:\n",
    "        continue\n",
    "\n",
    "      pred_score = pred_item_cf(user, song, user_song_playcount, user_played_songs, \n",
    "                song_sim_matrix, song_to_index, df_train)\n",
    "      rtn_songs.append((pred_score, song))\n",
    "\n",
    "    return sorted(rtn_songs, key=lambda x: x[0], reverse=True)\n",
    "\n",
    "\n",
    "def recommend_all_item_cf(df_train, df_val, rec_num):\n",
    "    \"\"\"\n",
    "    基于Item的CF方法: 根据训练集的数据，对所有校验集中的User推荐Song\n",
    "    :param df_train: 训练集\n",
    "    :param df_val: 校验集\n",
    "    :param rec_num: 为每个User推荐的song数量\n",
    "    :return: \n",
    "    \"\"\"\n",
    "    result_dict = dict()\n",
    "    \n",
    "    print(\"\\n开始初始化user_songs映射\")\n",
    "    time_start = time.time()\n",
    "    user_songs = init_user_songs(df_train)\n",
    "    print(\"初始化user_songs映射完成，耗时{:.4f}秒\".format(time.time() - time_start))\n",
    "    \n",
    "    print(\"\\n开始初始化song_users映射\")\n",
    "    time_start = time.time()\n",
    "    song_users = init_song_users(df_train)\n",
    "    print(\"初始化song_users映射完成，耗时{:.4f}秒\".format(time.time() - time_start))\n",
    "    \n",
    "    print(\"\\n开始初始化user_song_playcount映射\")\n",
    "    time_start = time.time()\n",
    "    user_song_playcount = init_user_song_playcount(df_train)\n",
    "    print(\"初始化user_song_playcount映射完成，耗时{:.4f}秒\".format(time.time() - time_start))\n",
    "\n",
    "    print(\"\\n开始初始化song_sim_matrix, song_to_index\")\n",
    "    time_start = time.time()\n",
    "    song_sim_matrix, song_to_index = init_song_sim_matrix(song_users)\n",
    "    print(\"初始化song_sim_matrix, song_to_index完成，耗时{:.4f}秒\".format(\n",
    "        time.time() - time_start))\n",
    "    \n",
    "    val_users = df_val[\"user\"].unique()\n",
    "    print(\"\\n开始为{}个来自校验集的user推荐songs\".format(len(val_users)))\n",
    "    time_start = time.time()\n",
    "    for i, user in enumerate(val_users):\n",
    "        if user not in user_songs:\n",
    "            print(\"{} is a new user.\".format(user))\n",
    "            continue\n",
    "\n",
    "        user_played_songs = set(user_songs.get(user, list()))\n",
    "        rec_songs = recommend_item_cf(user, song_users, user_song_playcount, user_played_songs,\n",
    "                      song_sim_matrix, song_to_index, df_train)\n",
    "        if len(rec_songs) > 0:\n",
    "            result_dict[user] = rec_songs[0: rec_num]\n",
    "\n",
    "        if (i + 1) % 100 == 0:\n",
    "            print(\"已经为{}个user计算好了推荐song的结果，耗时{:.4f}秒\".format(\n",
    "                i + 1, time.time() - time_start\n",
    "            ))\n",
    "\n",
    "    print(\"为{}个来自校验集的user推荐songs的工作已完成，耗时{:.4f}秒\".format(\n",
    "        len(val_users), time.time() - time_start\n",
    "    ))\n",
    "\n",
    "    return result_dict, len(song_users)\n",
    "\n",
    "\n",
    "def evaluate_item_cf(user_rec_dict, total_song_num, df_val):\n",
    "    \"\"\"\n",
    "    评估在校验集上的预测结果\n",
    "    :param user_rec_dict: 预测结果<user-推荐的song和评分列表>\n",
    "    :param total_song_num: 训练集中的song总数\n",
    "    :param df_val: 校验集\n",
    "    :return: \n",
    "    \"\"\"\n",
    "    n_tp = 0         # 校验集和预测结果匹配的song数\n",
    "    n_total_rec = 0      # 预测结果里的song数\n",
    "    n_total_real = 0      # 校验集里的song数\n",
    "    total_rec_songs = set()   # 预测结果里所有的song\n",
    "    rss_val = 0.0         # 预测结果与校验集结果的平方误差(用播放次数差来计算)\n",
    "    \n",
    "    unique_users = df_val[\"user\"].unique()\n",
    "    for user in unique_users:\n",
    "        real_songs = df_val.loc[df_val.user == user].song.values\n",
    "        n_total_real = n_total_real + len(real_songs)\n",
    "        \n",
    "        if user in user_rec_dict:\n",
    "            rec_songs = [x[1] for x in user_rec_dict[user]]\n",
    "            hit_songs = set(real_songs).intersection(set(rec_songs))\n",
    "        \n",
    "            total_rec_songs = total_rec_songs.union(hit_songs)\n",
    "            n_tp = n_tp + len(hit_songs)\n",
    "        \n",
    "            n_total_rec = n_total_rec + len(rec_songs)\n",
    "            \n",
    "            for pred_score, song in user_rec_dict[user]:\n",
    "                val_score = df_val.loc[df_val.user == user][df_val.song == song]\\\n",
    "                    .play_count.values\n",
    "                if len(val_score) > 0:\n",
    "                    rss_val = rss_val + (pred_score - val_score[0]) ** 2\n",
    "    \n",
    "    precision = 1.0 * n_tp / n_total_rec\n",
    "    recall = 1.0 * n_tp / n_total_real\n",
    "    coverage = 1.0 * len(total_rec_songs) / total_song_num\n",
    "    rmse = np.sqrt(1.0 * rss_val / df_val.shape[0])\n",
    "    \n",
    "    print(\"评价结束: precision={:.4f}, recall={:.4f}, coverage={:.4f}, rmse={:.4f}\".format(\n",
    "        precision, recall, coverage, rmse\n",
    "    ))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "id": "ACK9nHvcNArp",
    "colab_type": "code",
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 523.0
    },
    "outputId": "6a875b0b-07a7-44a3-8add-a92dbf0c2246"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n开始初始化user_songs映射\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "初始化user_songs映射完成，耗时0.2040秒\n\n开始初始化song_users映射\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "初始化song_users映射完成，耗时0.2720秒\n\n开始初始化user_song_playcount映射\n初始化user_song_playcount映射完成，耗时0.0200秒\n\n开始初始化song_sim_matrix, song_to_index\n初始化song_sim_matrix, song_to_index完成，耗时3.9442秒\n\n开始为726个来自校验集的user推荐songs\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "已经为100个user计算好了推荐song的结果，耗时5.6333秒\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "已经为200个user计算好了推荐song的结果，耗时11.1656秒\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "已经为300个user计算好了推荐song的结果，耗时15.0129秒\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "已经为400个user计算好了推荐song的结果，耗时18.3861秒\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "已经为500个user计算好了推荐song的结果，耗时21.5922秒\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "已经为600个user计算好了推荐song的结果，耗时24.1414秒\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "67c5b5b1982902d15badd8ce0c18b3278ec4bfc0 is a new user.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "62420be0fd0df5ab0eb4cba35a4bc7cb3e3b506a is a new user.\n3ab78e39bddeaeb789edad041fff03050077417c is a new user.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "已经为700个user计算好了推荐song的结果，耗时25.5635秒\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "为726个来自校验集的user推荐songs的工作已完成，耗时25.8255秒\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Python37\\lib\\site-packages\\ipykernel_launcher.py:145: UserWarning: Boolean Series key will be reindexed to match DataFrame index.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "评价结束: precision=0.0306, recall=0.0295, coverage=0.1950, rmse=3.7606\n"
     ]
    }
   ],
   "source": [
    "# 为每个校验集中的User推荐10首Song\n",
    "result_dict, total_song_num = recommend_all_item_cf(data_train, data_val, 10)\n",
    "# 评价推荐结果\n",
    "evaluate_item_cf(result_dict, total_song_num, data_val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "id": "AbIpfSGYN7jn",
    "colab_type": "code",
    "colab": {}
   },
   "outputs": [],
   "source": [
    "# 基于矩阵分解的CF\n",
    "import numpy as np\n",
    "from numpy.random import random\n",
    "import time\n",
    "\n",
    "def init_user_index(df_train):\n",
    "    \"\"\"\n",
    "    初始化user对应的索引\n",
    "    :param df_train: \n",
    "    :return: \n",
    "    \"\"\"\n",
    "    user_index = dict()\n",
    "    for idx, user in enumerate(df_train[\"user\"].unique()):\n",
    "        user_index[user] = idx\n",
    "    return user_index\n",
    "\n",
    "\n",
    "def init_song_index(df_train):\n",
    "    \"\"\"\n",
    "    初始化song对应的索引\n",
    "    :param df_train: \n",
    "    :return: \n",
    "    \"\"\"\n",
    "    song_index = dict()\n",
    "    for idx, song in enumerate(df_train[\"song\"].unique()):\n",
    "        song_index[song] = idx\n",
    "    return song_index\n",
    "\n",
    "\n",
    "class SVDModel:\n",
    "    def __init__(self, df_train, n_users, n_songs):\n",
    "        \"\"\"\n",
    "        构造函数\n",
    "        :param df_train:  训练集数据\n",
    "        :param n_users:  训练集里的独立用户数\n",
    "        :param n_songs:  训练集里的独立歌曲数\n",
    "        \"\"\"\n",
    "        self.K = 50\n",
    "        self.gamma = 0.0001\n",
    "        self.Lambda = 0.5\n",
    "\n",
    "        self.mu = df_train[\"play_count\"].mean()\n",
    "        self.bi = np.zeros((n_songs, 1), dtype=np.float)\n",
    "        self.bu = np.zeros((n_users, 1), dtype=np.float)\n",
    "        self.qi = np.zeros((n_songs, self.K), dtype=np.float)\n",
    "        self.pu = np.zeros((n_users, self.K), dtype=np.float)\n",
    "\n",
    "        for each_uidx in range(n_users):\n",
    "            self.pu[each_uidx] = np.reshape(random((self.K, 1)) / 10 * np.sqrt(self.K), \n",
    "                                            self.K)\n",
    "\n",
    "        for each_iidx in range(n_songs):\n",
    "            self.qi[each_iidx] = np.reshape(random((self.K, 1)) / 10 * np.sqrt(self.K), \n",
    "                                            self.K)\n",
    "\n",
    "    def pred_score(self, uidx, iidx):\n",
    "        \"\"\"\n",
    "        根据当前超参数计算uidx对iidx的打分\n",
    "        打分预测模型为 score(uidx, iidx) = mu + bi[iidx] + bu[uidx] + np.sum(qi[iidx] * pu[uidx])\n",
    "        :param uidx: 用户索引\n",
    "        :param iidx: 歌曲索引\n",
    "        \"\"\"\n",
    "        rtn_score = self.mu + self.bi[iidx] + self.bu[uidx] \\\n",
    "                    + np.sum(self.qi[iidx] * self.pu[uidx])\n",
    "        return rtn_score\n",
    "\n",
    "\n",
    "def train_model(df_train):\n",
    "    \"\"\"\n",
    "    训练模型\n",
    "    :param df_train: \n",
    "    :return: \n",
    "    \"\"\"\n",
    "    n_users = len(df_train[\"user\"].unique())\n",
    "    n_songs = len(df_train[\"song\"].unique())\n",
    "    model = SVDModel(df_train, n_users, n_songs)\n",
    "\n",
    "    # 训练的epoch数量\n",
    "    epoches = 50\n",
    "\n",
    "    n_train_records = df_train.shape[0]\n",
    "    user_index = init_user_index(df_train)\n",
    "    song_index = init_song_index(df_train)\n",
    "\n",
    "    time_start = time.time()\n",
    "    print(\"开始执行{}个epoch的模型训练，使用随机梯度下降法\".format(epoches))\n",
    "    for epoch in range(epoches):\n",
    "        each_rmse_sum = 0.0\n",
    "        kk = np.random.permutation(n_train_records)\n",
    "        \n",
    "        for j in range(n_train_records):\n",
    "            line = kk[j]\n",
    "            \n",
    "            uidx = user_index[df_train.iloc[line][\"user\"]]\n",
    "            iidx = song_index[df_train.iloc[line][\"song\"]]\n",
    "            score = df_train.iloc[line][\"play_count\"]\n",
    "\n",
    "            each_pred_score = model.pred_score(uidx, iidx)\n",
    "            eui = score - each_pred_score\n",
    "            each_rmse_sum = each_rmse_sum + eui**2\n",
    "            \n",
    "            model.bi[iidx] = model.bi[iidx] \\\n",
    "                             + model.gamma * (eui - model.Lambda * model.bi[iidx])\n",
    "            model.bu[uidx] = model.bu[uidx] \\\n",
    "                             + model.gamma * (eui - model.Lambda * model.bu[uidx])\n",
    "    \n",
    "            temp = model.qi[iidx]\n",
    "            model.qi[iidx] = model.qi[iidx] \\\n",
    "                             + model.gamma * (eui * model.pu[uidx] \n",
    "                                              - model.Lambda * model.qi[iidx])\n",
    "            model.pu[uidx] = model.pu[uidx] \\\n",
    "                             + model.gamma * (eui * temp \n",
    "                                              - model.Lambda * model.pu[uidx])\n",
    "\n",
    "        model.gamma = model.gamma * 0.93\n",
    "        each_rmse = np.sqrt(each_rmse_sum / n_train_records)\n",
    "\n",
    "        print(\"完成了第{}个epoch的训练，累计耗时{:.4f}秒, each_rmse={}\".format(\n",
    "            epoch + 1, time.time() - time_start, each_rmse))\n",
    "        \n",
    "    print(\"执行{}个epoch的模型训练已完成，总耗时{:.4f}秒\".format(\n",
    "        epoches, time.time() - time_start))\n",
    "\n",
    "    return model\n",
    "\n",
    "\n",
    "def recommend_all_svd_cf(model, df_train, df_val, rec_num):\n",
    "    \"\"\"\n",
    "    利用在训练集里训练好的模型来对校验集里的user推荐songs\n",
    "    :param model: 已经训练好的模型\n",
    "    :param df_train: 训练集数据\n",
    "    :param df_val: 校验集数据\n",
    "    :param rec_num: 对每个user推荐的结果数\n",
    "    \"\"\"\n",
    "    user_rec_dict = dict()\n",
    "    \n",
    "    time_start = time.time()\n",
    "    user_index = init_user_index(df_train)\n",
    "    print(\"\\n\\n初始化user_index耗时{:.4f}秒\".format(time.time() - time_start))\n",
    "    \n",
    "    time_start = time.time()\n",
    "    song_index = init_song_index(df_train)\n",
    "    print(\"\\n\\n初始化song_index耗时{:.4f}秒\".format(time.time() - time_start))\n",
    "    \n",
    "    time_start = time.time()\n",
    "    val_users = df_val[\"user\"].unique()\n",
    "    print(\"\\n\\n获得去重后的校验集user耗时{:.4f}秒\".format(time.time() - time_start))\n",
    "    \n",
    "    print(\"\\n\\n开始为{}个来自校验集的user推荐songs\".format(len(val_users)))\n",
    "    time_start = time.time()\n",
    "    \n",
    "    count = 0\n",
    "    for user in val_users:\n",
    "        count = count + 1\n",
    "        if user not in user_index:\n",
    "            print(\"第{}个user: {} is a new user.\".format(count, user))\n",
    "            continue\n",
    "    \n",
    "        uidx = user_index[user]\n",
    "        each_df = pd.DataFrame(columns=[\"item_id\", \"score\"])\n",
    "        for song, iidx in song_index.items():\n",
    "            each_score = model.pred_score(uidx, iidx)\n",
    "            each_df.loc[len(each_df)] = [song, each_score]\n",
    "        \n",
    "        each_df.sort_values(by=\"score\", ascending=False)\n",
    "        user_rec_dict[user] = each_df\n",
    "    \n",
    "        if count % 100 == 0:\n",
    "            print(\"\\n已经为{}个user推荐了song, 累计耗时{:.4f}秒\".format(\n",
    "                count, time.time() - time_start\n",
    "            ))\n",
    "    \n",
    "    return user_rec_dict, len(song_index.keys())\n",
    "\n",
    "\n",
    "def evaluate_svd_cf(user_rec_dict, total_song_num, df_val, rec_num):\n",
    "    \"\"\"\n",
    "    评估在校验集上的预测结果\n",
    "    :param user_rec_dict: 对校验集里的每个user的推荐结果\n",
    "    :param total_song_num: 训练集中的song总数\n",
    "    :param df_val: 校验集\n",
    "    :param rec_num: 为每个user推荐的song数量\n",
    "    :return: \n",
    "    \"\"\"\n",
    "    n_tp = 0         # 校验集和预测结果匹配的song数\n",
    "    n_total_rec = 0      # 预测结果里的song数\n",
    "    n_total_real = 0      # 校验集里的song数\n",
    "    total_rec_songs = set()   # 预测结果里所有的song\n",
    "    rss_val = 0.0         # 预测结果与校验集结果的平方误差(用播放次数差来计算)\n",
    "  \n",
    "    unique_users = df_val[\"user\"].unique()\n",
    "    print(\"开始对{}个user的推荐结果进行效果评估\".format(len(unique_users)))\n",
    "    count = 0\n",
    "    time_start = time.time()\n",
    "    for user in unique_users:\n",
    "        count = count + 1\n",
    "        real_songs = df_val.loc[df_val.user == user].song.values\n",
    "        n_total_real = n_total_real + len(real_songs)\n",
    "    \n",
    "        if user in user_rec_dict:\n",
    "            rec_results = user_rec_dict[user].iloc[0:rec_num]\n",
    "            rec_songs = rec_results[\"item_id\"].values\n",
    "            hit_songs = set(real_songs).intersection(set(rec_songs))\n",
    "    \n",
    "            total_rec_songs = total_rec_songs.union(hit_songs)\n",
    "            n_tp = n_tp + len(hit_songs)\n",
    "      \n",
    "            n_total_rec = n_total_rec + len(rec_songs)\n",
    "      \n",
    "            for song, pred_score in rec_results[[\"item_id\", \"score\"]].values:\n",
    "                val_score = df_val.loc[df_val.user == user][df_val.song == song]\n",
    "                val_score = val_score.play_count.values\n",
    "                if len(val_score) > 0:\n",
    "                    rss_val = rss_val + (pred_score - val_score[0]) ** 2\n",
    "            \n",
    "            if count % 100 == 0:\n",
    "                print(\"已经完成对{}个user的推荐结果的效果评估，累计耗时{:.4f}秒\".format(\n",
    "                    count, time.time() - time_start\n",
    "                ))\n",
    "            \n",
    "    precision = 1.0 * n_tp / n_total_rec\n",
    "    recall = 1.0 * n_tp / n_total_real\n",
    "    coverage = 1.0 * len(total_rec_songs) / total_song_num\n",
    "    rmse = np.sqrt(1.0 * rss_val / df_val.shape[0])\n",
    "  \n",
    "    print(\"评价结束: precision={:.4f}, recall={:.4f}, coverage={:.4f}, rmse={}\".format(\n",
    "      precision, recall, coverage, rmse\n",
    "    ))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "id": "a02AVczPfzXi",
    "colab_type": "code",
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000.0
    },
    "outputId": "402a4fbc-388b-4958-987b-970f6b69cd08",
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "开始执行50个epoch的模型训练，使用随机梯度下降法\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "完成了第1个epoch的训练，累计耗时19.5461秒, each_rmse=[35.57689327]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "完成了第2个epoch的训练，累计耗时38.5862秒, each_rmse=[35.34301241]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "完成了第3个epoch的训练，累计耗时57.6683秒, each_rmse=[35.12961385]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "完成了第4个epoch的训练，累计耗时76.6924秒, each_rmse=[34.90811454]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "完成了第5个epoch的训练，累计耗时95.7355秒, each_rmse=[34.65573812]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "完成了第6个epoch的训练，累计耗时114.8526秒, each_rmse=[34.34973964]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "完成了第7个epoch的训练，累计耗时133.9017秒, each_rmse=[33.96760937]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "完成了第8个epoch的训练，累计耗时152.9237秒, each_rmse=[33.49204335]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "完成了第9个epoch的训练，累计耗时171.9048秒, each_rmse=[32.91483551]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "完成了第10个epoch的训练，累计耗时190.9399秒, each_rmse=[32.24707098]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "完成了第11个epoch的训练，累计耗时210.0310秒, each_rmse=[31.50958704]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "完成了第12个epoch的训练，累计耗时229.0781秒, each_rmse=[30.74444008]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "完成了第13个epoch的训练，累计耗时248.2062秒, each_rmse=[29.99741313]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "完成了第14个epoch的训练，累计耗时267.2323秒, each_rmse=[29.31211516]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "完成了第15个epoch的训练，累计耗时286.2814秒, each_rmse=[28.70106083]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "完成了第16个epoch的训练，累计耗时305.3565秒, each_rmse=[28.19182418]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "完成了第17个epoch的训练，累计耗时324.4216秒, each_rmse=[27.77340557]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "完成了第18个epoch的训练，累计耗时343.3766秒, each_rmse=[27.42959861]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "完成了第19个epoch的训练，累计耗时362.3667秒, each_rmse=[27.15230504]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "完成了第20个epoch的训练，累计耗时381.4178秒, each_rmse=[26.92452696]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "完成了第21个epoch的训练，累计耗时400.3959秒, each_rmse=[26.73748651]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "完成了第22个epoch的训练，累计耗时419.3740秒, each_rmse=[26.5861529]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "完成了第23个epoch的训练，累计耗时438.3551秒, each_rmse=[26.44940947]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "完成了第24个epoch的训练，累计耗时457.3552秒, each_rmse=[26.33545701]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "完成了第25个epoch的训练，累计耗时476.3932秒, each_rmse=[26.23694448]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "完成了第26个epoch的训练，累计耗时495.4023秒, each_rmse=[26.14678464]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "完成了第27个epoch的训练，累计耗时514.4344秒, each_rmse=[26.06910935]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "完成了第28个epoch的训练，累计耗时533.5245秒, each_rmse=[25.99805203]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "完成了第29个epoch的训练，累计耗时552.6646秒, each_rmse=[25.93128427]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "完成了第30个epoch的训练，累计耗时571.7387秒, each_rmse=[25.87324811]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "完成了第31个epoch的训练，累计耗时590.7658秒, each_rmse=[25.81899449]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "完成了第32个epoch的训练，累计耗时609.7459秒, each_rmse=[25.76830578]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "完成了第33个epoch的训练，累计耗时628.8100秒, each_rmse=[25.72228807]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "完成了第34个epoch的训练，累计耗时647.7851秒, each_rmse=[25.68042409]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "完成了第35个epoch的训练，累计耗时666.7911秒, each_rmse=[25.64078421]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "完成了第36个epoch的训练，累计耗时685.8072秒, each_rmse=[25.60437379]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "完成了第37个epoch的训练，累计耗时704.7823秒, each_rmse=[25.5705404]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "完成了第38个epoch的训练，累计耗时723.7954秒, each_rmse=[25.53892221]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "完成了第39个epoch的训练，累计耗时742.8975秒, each_rmse=[25.50980381]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "完成了第40个epoch的训练，累计耗时761.8736秒, each_rmse=[25.48246819]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "完成了第41个epoch的训练，累计耗时780.8777秒, each_rmse=[25.45751706]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "完成了第42个epoch的训练，累计耗时799.9518秒, each_rmse=[25.43420195]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "完成了第43个epoch的训练，累计耗时818.9538秒, each_rmse=[25.41189863]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "完成了第44个epoch的训练，累计耗时837.9579秒, each_rmse=[25.39190073]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "完成了第45个epoch的训练，累计耗时856.9270秒, each_rmse=[25.37296166]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "完成了第46个epoch的训练，累计耗时875.9531秒, each_rmse=[25.35514464]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "完成了第47个epoch的训练，累计耗时894.9802秒, each_rmse=[25.33886001]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "完成了第48个epoch的训练，累计耗时913.9903秒, each_rmse=[25.32375782]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "完成了第49个epoch的训练，累计耗时933.0624秒, each_rmse=[25.30962149]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "完成了第50个epoch的训练，累计耗时952.0995秒, each_rmse=[25.29638183]\n执行50个epoch的模型训练已完成，总耗时952.1005秒\n\n\n初始化user_index耗时0.0040秒\n\n\n初始化song_index耗时0.0030秒\n\n\n获得去重后的校验集user耗时0.0010秒\n\n\n开始为726个来自校验集的user推荐songs\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n已经为100个user推荐了song, 累计耗时155.1129秒\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n已经为200个user推荐了song, 累计耗时314.6250秒\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n已经为300个user推荐了song, 累计耗时478.8254秒\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n已经为400个user推荐了song, 累计耗时638.6405秒\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n已经为500个user推荐了song, 累计耗时804.6760秒\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n已经为600个user推荐了song, 累计耗时968.0424秒\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "第633个user: 67c5b5b1982902d15badd8ce0c18b3278ec4bfc0 is a new user.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "第667个user: 62420be0fd0df5ab0eb4cba35a4bc7cb3e3b506a is a new user.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "第670个user: 3ab78e39bddeaeb789edad041fff03050077417c is a new user.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n已经为700个user推荐了song, 累计耗时1127.1395秒\n"
     ]
    }
   ],
   "source": [
    "# 在训练集上训练模型\n",
    "my_model = train_model(data_train)\n",
    "# 为每个校验集中的User推荐10首Song\n",
    "result_dict, total_song_num = recommend_all_svd_cf(my_model, data_train, data_val, 10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "开始对726个user的推荐结果进行效果评估\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Python37\\lib\\site-packages\\ipykernel_launcher.py:211: UserWarning: Boolean Series key will be reindexed to match DataFrame index.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "已经完成对100个user的推荐结果的效果评估，累计耗时3.5782秒\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "已经完成对200个user的推荐结果的效果评估，累计耗时6.7034秒\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "已经完成对300个user的推荐结果的效果评估，累计耗时9.8036秒\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "已经完成对400个user的推荐结果的效果评估，累计耗时12.9287秒\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "已经完成对500个user的推荐结果的效果评估，累计耗时16.0269秒\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "已经完成对600个user的推荐结果的效果评估，累计耗时19.1401秒\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "已经完成对700个user的推荐结果的效果评估，累计耗时22.1683秒\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "评价结束: precision=0.0206, recall=0.0199, coverage=0.0125, rmse=[2.53399739]\n"
     ]
    }
   ],
   "source": [
    "# 评价推荐结果\n",
    "evaluate_svd_cf(result_dict, total_song_num, data_val, 10)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "总结:\n",
    "1. 基于用户\n",
    "提前计算用户相似度: 3.4462秒\n",
    "推荐: 36.3211秒\n",
    "评价: precision=0.0296, recall=0.0285, coverage=0.0650, rmse=11.5904\n",
    "\n",
    "2. 基于物品\n",
    "提前计算物品相似度: 3.9442秒\n",
    "推荐: 25.8255秒\n",
    "评价: precision=0.0306, recall=0.0295, coverage=0.1950, rmse=3.7606\n",
    "\n",
    "3. 基于矩阵分解:\n",
    "模型训练: 952.1005秒\n",
    "推荐：1127.1395秒\n",
    "评价： precision=0.0206, recall=0.0199, coverage=0.0125, rmse=[2.53399739]\n",
    "\n",
    "分析:\n",
    "\n",
    "+ 从运行速度上看，基于用户或物品的相似度的CF，对于较小的数据集，可以提前算好相似度矩阵，进而大大提高后续推荐过程的速度;\n",
    "而基于矩阵分解需要提前训练模型，这一步颇为耗时，而且对于数据较大的情形，给所有用户进行推荐的过程也开始变慢；\n",
    "+ 但从评价结果上看，基于矩阵分解带来了更高的precision和更低的rmse， 但recall和coverage并未带来改善。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "colab": {
   "name": "recommend_music.ipynb",
   "provenance": [],
   "collapsed_sections": []
  },
  "kernelspec": {
   "name": "python3",
   "display_name": "Python 3"
  },
  "accelerator": "GPU"
 },
 "nbformat": 4,
 "nbformat_minor": 0
}
